'''
Race_Statistics_Functions.py

This module contains the race stats function generate_race_stats(). 
It is called in Race_Detection_and_Statistics.py to compute race stats.                      
'''
from collections import OrderedDict
import numpy as np
import pandas as pd

from .Race_Msg_Outcome import get_msg_outcome

######################
## Main Functions ##
######################

def generate_race_stats(date, sym, msgs, top, depth, race_recs, ticktable, price_factor, \
                           race_param):
    '''
    This function generates single level race stats for a given symdate after race detection.
    This function loops over each row in race record (output of race detection code) and calls 
    find_single_lvl_races() on each row to generate stats for each race. 
    
    Param: Please refer to Section 10.5 of the Code and Data Appendix.
        msgs: df of msgs
        top: top of book df, generated by order book construction
        depth: output of order book construction
        race_recs: output of race detection code
        ticktable: df, tick size info for different prices
        price_factor: constant, unit conversion
        sym, date: symbol-date
        race_param: dict of race parameters
        
    Return:
        stats: df with all race stats for the symbol-date, one row per race
    '''
    ##### Initialization
    stats = {}
    strict_fail = race_param['strict_fail']
    
    ##### Get ME (outbound msgs) dataframes, set index to be UniqueOrderID-EventNum pairs and set them as global variables
    # We define those global variables to efficiently find the outbounds associated with the race inbounds
    me_cols = ['UniqueOrderID', 'EventNum', 'MessageType', 'ExecType', 'UnifiedMessageType', 'LeavesQty', 'ExecutedPrice', 'ExecutedQty']
    me = msgs.loc[msgs['EventNum'].notnull() & ((msgs['MessageType'] == 'Execution_Report') | (msgs['MessageType'] == 'Cancel_Reject')), me_cols].copy()
    me = me.reset_index()
    me = me.set_index(['UniqueOrderID', 'EventNum'])
    me = me.sort_index(level=0)

    me_qr_bid_cols = ['UniqueOrderID', 'BidEventNum', 'MessageType', 'ExecType', 'UnifiedMessageType', 'LeavesQty', 'ExecutedPrice', 'ExecutedQty']
    me_qr_bid = msgs.loc[msgs['BidEventNum'].notnull() & ((msgs['MessageType'] == 'Execution_Report') | (msgs['MessageType'] == 'Cancel_Reject')), me_qr_bid_cols].copy()
    me_qr_bid = me_qr_bid.reset_index()
    me_qr_bid = me_qr_bid.set_index(['UniqueOrderID', 'BidEventNum'])
    me_qr_bid = me_qr_bid.sort_index(level=0)

    me_qr_ask_cols = ['UniqueOrderID', 'AskEventNum', 'MessageType', 'ExecType', 'UnifiedMessageType', 'LeavesQty', 'ExecutedPrice', 'ExecutedQty']
    me_qr_ask = msgs.loc[msgs['AskEventNum'].notnull() & ((msgs['MessageType'] == 'Execution_Report') | (msgs['MessageType'] == 'Cancel_Reject')), me_qr_ask_cols].copy()
    me_qr_ask = me_qr_ask.reset_index()
    me_qr_ask = me_qr_ask.set_index(['UniqueOrderID', 'AskEventNum'])
    me_qr_ask = me_qr_ask.sort_index(level=0)
    
    ##### Loop over races to populate statistics
    for ix, race in race_recs.iterrows():
        stats[ix] = stats_for_one_race(race, msgs, me, me_qr_ask, me_qr_bid, top, depth,\
                                   ticktable, price_factor, \
                                   strict_fail)
    ##### construct pd.DataFrame
    stats = pd.DataFrame.from_dict(stats, orient='index')
    stats['Date'] = date
    stats['Symbol'] = sym
    return stats
    
def stats_for_one_race(race, msgs, me, me_qr_ask, me_qr_bid, top, depth,\
                   ticktable, price_factor, strict_fail):
    '''
    This function generates stats for a singlelvl race
    
    Param: Please refer to Section 10.3 of the Code and Data Appendix.
        race: pd.Series, the record for one specific race from race_recs
        msgs: df of msgs
        me, me_qr_ask, me_qr_bid: dfs of outbound msgs for non-quotes and quotes
        top: top of book df, generated by order book construction
        depth: depth generated by order book construction
        ticktable: df, tick size info for different prices
        price_factor: constant for price unit conversion
        strict_fail: strict fail definition. If True, only expired IOCs are considered as fails
        
    Return:
        out: pd.Series with all relevant statistics of the race
    '''
    ###########################################################################
    ### Specify constant
    seconds_to_microseconds = 1e6

    ### Initialize dictionary for output
    out = OrderedDict()

    ### Extract basic information about the race
    S, P_Signed, ix_st, time_st, race_horizon = race[['Side','P_Signed','Race_Start_Idx','MessageTimestamp','Race_Horizon']]

    ### Get race messages
    race_msgs = msgs.loc[race['Race_Msgs_Idx']]
    race_msgs['Idx'] = race['Race_Msgs_Idx']
    proc_time = race_msgs['ProcessingTime'].values
    
    ### Create an unsigned version of price level
    Sign = 1 * (S == 'Ask') - 1 * (S == 'Bid')
    P = Sign * P_Signed

    ### Create pointer to top-of-book as-of the first message in the race
    top_1 = top.loc[ix_st]
    out['MidPt'] = top_1['MidPt'] / price_factor

    ###########################################################################
    ##### General variables 
    ### Race IDs 
    out['SingleLvlRaceID'] = race['SingleLvlRaceID']
    out['Race_Start_Idx'] = ix_st
    out['Race_Msgs_Idx'] = race['Race_Msgs_Idx']

    # The Info Horizon length. 
    # For 'Info_Horizon' method this is the same as Race_Horizon
    # For 'Fixed_Horizon' method, this is not the the same as Race_Horizon
    out['Race_Horizon'] = race_horizon.total_seconds() * seconds_to_microseconds

    # processing time of the first message
    out['Processing_Time'] = pd.Timedelta(proc_time[0]).total_seconds() * seconds_to_microseconds

    ### Top of book info
    out['BestBid'] = top_1['BestBid'] / price_factor
    out['BestBidQty'] = top_1['BestBidQty']
    out['BestAsk'] = top_1['BestAsk'] / price_factor
    out['BestAskQty'] = top_1['BestAskQty'] 

    ### RaceID, Side, Price, and Depth at start of race
    out['Side'] = S
    out['RacePrice'] = P / price_factor
    out['TickSize'] = ticktable.loc[ticktable['p_int64'] <= P, 'tick'].iloc[-1].item()
    
    ### Near Boundary
    # Flag races within 10 ticks of the boundary between tick levels
    tick_gr = (1./price_factor) * P + 5 * out['TickSize']
    tick_less = max((1./price_factor) * P - 5 * out['TickSize'], 0)
    out['NearTickBoundary'] = (ticktable.loc[ticktable['p'] <= tick_gr, 'tick'].iloc[-1].item() > out['TickSize']) \
                               or (ticktable.loc[ticktable['p'] <= tick_less, 'tick'].iloc[-1].item() < out['TickSize'])  
    
    ### Depth
    out['Depth_Disp'] = GetDepth(S, P, ix_st, 'Displayed', depth)
    out['Depth_Total'] = GetDepth(S, P, ix_st, 'Total', depth)
    
    ### Number of race relevant No Response Msgs
    out['M_RaceRlvtNoResponse'] = race_msgs['%sRaceRlvtNoResponse' % S].sum()
    
    ### Write RaceRlvtOutcome with the msg outcome accounting for takes that are price dependent
    # Note that Outcome Classification is done twice: here and previously in race detection.
    # This is because we don't save the result of the outcome calculation.
    race_msgs['%sRaceMsgOutcome' % S] = get_msg_outcome(S, P_Signed, race_msgs, strict_fail)
    
    ###########################################################################
    ##### Race timings 
    ### Time M1 and Time MLast
    out['Time_M1'] = race_msgs['MessageTimestamp'].iloc[0]
    out['Time_MLast'] = race_msgs['MessageTimestamp'].iloc[-1]
    ### Time from Success to Fail
    Time_S1 = race_msgs.loc[(race_msgs['%sRaceMsgOutcome' % S] == 'Success'), 'MessageTimestamp'].iloc[0]
    Time_F1 = race_msgs.loc[(race_msgs['%sRaceMsgOutcome' % S] == 'Fail'), 'MessageTimestamp'].iloc[0]
    out['Time_S1_F1'] = (Time_F1 - Time_S1).total_seconds() * seconds_to_microseconds
    out['Time_S1_F1_Max_0'] = max(out['Time_S1_F1'], 0.)
    out['Time_M1_S1'] = (Time_S1 - out['Time_M1']).total_seconds() * seconds_to_microseconds
    out['Time_M1_F1'] = (Time_F1 - out['Time_M1']).total_seconds() * seconds_to_microseconds
    
    ###########################################################################
    ##### Shares traded or canceled in race and number of trades
    # Note:  max(0, x) prevents missing values to be introduced 
    # (i.e. if there is no quantity cancelled, we record 0, not missing.)

    ### Quantity traded and # trades in the race.
    # Use the TradeNum_Vol() to get qty traded for trade in the race. 
    # Only include depth at the race Price/Side.
    # (See the code block for the function for additional detail)    
    at_P = True
    out['Qty_Traded'], out['Num_Trades'] = race_msgs.apply(TradeNum_Vol, args=(S, P, at_P, me, me_qr_ask, me_qr_bid), axis=1).sum()
    out['Qty_Traded'] = max(0, out['Qty_Traded']) 
    out['Num_Trades'] = max(0, out['Num_Trades'])
    out['Value_Traded'] = out['Qty_Traded'] * out['RacePrice']

    ### Quantity canceled
    # Note that all cancels that have any cancelled quantity are labelled success. 
    out['Qty_Cancelled'] = max(0,race_msgs.loc[(race_msgs['%sRaceRlvtType' % S] == 'Cancel Attempt') & (race_msgs['%sRaceMsgOutcome' % S] =='Success'), '%sRaceRlvtQty' %S].sum())
    out['Value_Cancelled'] = out['Qty_Cancelled'] * out['RacePrice']

    ### Active Qty: Qty_Traded at P + Qty_Cancelled at P
    out['Qty_Active'] = out['Qty_Traded'] + out['Qty_Cancelled']
    
    ### Quantity of shares remaining = total depth - qty canceled and qty traded
    out['Qty_Remaining'] = max(0, out['Depth_Total'] - out['Qty_Cancelled'] - out['Qty_Traded'])
    
    ### Quantity of displayed depth that is remaining (if there was hidden depth, set it to 0)
    out['Qty_Remaining_Disp'] = max(0, out['Depth_Disp'] - out['Qty_Cancelled'] - out['Qty_Traded'])
    
    ### % of depth remaining after the race
    # We assert for zero division. In theory, Depth_Disp should not be zero given there is a race.
    # In practice, Depth_Disp might be zero (rarely) because our order book construction is not perfect.
    out['Qty_Remaining_Disp_Pct'] = np.float64(out['Qty_Remaining_Disp'])/np.float64(out['Depth_Disp']) if out['Depth_Disp'] != 0. else np.nan
                                      
    ###########################################################################
    ##### Number of users/messages/firms (N/M/F) in Race Relevant Msgs With T us of the race starting msg
    
    ### Preparation
    # Slice relevant messages that are close in time and are cancels/takes at P or takes worse than P (deeper in the book).
    # time_st and race_horizon come from the race rec data
    time_st = time_st.to_datetime64()
    time_end = (time_st + race_horizon).to_datetime64()
    t_close =  np.timedelta64(10, 'ms')
    is_close_in_time = (msgs['MessageTimestamp'].to_numpy() >= (time_st - t_close)) \
                        & (msgs['MessageTimestamp'].to_numpy() <= (time_st + t_close))
    RaceRlvt = msgs['%sRaceRlvt' % S].to_numpy()
    RaceRlvtType = msgs['%sRaceRlvtType' % S].to_numpy()
    RaceRlvtPriceLvlSigned = msgs['%sRaceRlvtPriceLvlSigned' % S].to_numpy()
    is_canc_eq_p = is_close_in_time & RaceRlvt & (RaceRlvtType == 'Cancel Attempt') & (RaceRlvtPriceLvlSigned == P_Signed)
    is_take_eq_p = is_close_in_time & RaceRlvt & (RaceRlvtType == 'Take Attempt') & (RaceRlvtPriceLvlSigned == P_Signed)
    is_take_deeper_p = is_close_in_time & RaceRlvt & (RaceRlvtType == 'Take Attempt') & (RaceRlvtPriceLvlSigned > P_Signed)
    rel_msgs = msgs.loc[is_canc_eq_p | is_take_eq_p | is_take_deeper_p, 
                        ['MessageTimestamp', 'UserID', '%sRaceRlvtType' % S,
                         '%sRaceRlvtPriceLvlSigned' % S, 'UnifiedMessageType',
                         'Event', 'BidRaceRlvtNoResponse', 'AskRaceRlvtNoResponse',
                         '%sRaceRlvtOutcomeGroup' % S, '%sRaceRlvtBestExecPriceLvlSigned' % S,'TIF', 'FirmID']]
    # Generate outcome
    rel_msgs['Outcome'] = get_msg_outcome(S, P_Signed, rel_msgs, strict_fail)

    ### Number of users/messages/firms (N/M/F) in Race Relevant Msgs
    # within T from the start of race for T = 50us, 100us, 200us, 500us, 1ms
    # N_Within_1000us, M_Canc_Within_1000us and M_Prior_1000us are all used for the filters.
    # Note that this section counts all race relevant msgs (no matter whether they are in this race)
    # All vars in this section are named by *_Within_Ts.
    # The vars in the next section named N/M_Fail/Success Canc/Take, etc. are counting race msgs (only msgs of this race)
    is_take = rel_msgs['%sRaceRlvtType' % S].to_numpy() == 'Take Attempt' # including takes at P and deeper than P
    is_canc = (rel_msgs['%sRaceRlvtType' % S].to_numpy() == 'Cancel Attempt') \
              & (rel_msgs['%sRaceRlvtPriceLvlSigned' % S].to_numpy() == P_Signed)
    is_fail = rel_msgs['Outcome'].to_numpy() == 'Fail'
    is_success = rel_msgs['Outcome'].to_numpy() == 'Success'
    is_IOC = (rel_msgs['UnifiedMessageType'].to_numpy() == 'Gateway New Order (IOC)') & (rel_msgs['TIF'].to_numpy() == 'IOC')
    is_Lim = rel_msgs['UnifiedMessageType'].to_numpy() == 'Gateway New Order (Limit)'
    T_list = ['50', '100', '200', '500', '1000', '2000', '3000']
    for T in T_list:
        in_T = (rel_msgs['MessageTimestamp'].to_numpy() >= time_st) \
                & (rel_msgs['MessageTimestamp'].to_numpy() <= (time_st + np.timedelta64(int(T), 'us')))
        ## Number of Messages (M)
        out['M_Within_%sus' % T] = in_T.sum()
        out['M_Canc_Within_%sus' % T] = (in_T & is_canc).sum()
        out['M_Take_Within_%sus' % T] = (in_T & is_take).sum()
        out['M_Fail_Within_%sus' % T] = (in_T & is_fail).sum()
        out['M_Success_Within_%sus' % T] = (in_T & is_success).sum()
        out['M_Take_IOC_Within_%sus' % T] = (is_take & is_IOC & in_T).sum()
        out['M_Take_Lim_Within_%sus' % T] = (is_take & is_Lim & in_T).sum()

        ## Number of Users (N)
        out['N_Within_%sus' % T] = rel_msgs.loc[in_T, 'UserID'].nunique() 

        ### The number of firms within T
        # For some messages we don't know their FirmID (firm unknown)
        # Each of those unknown firms are all treated as distinct firms
        out['F_Within_%sus' % T] = rel_msgs.loc[in_T, 'FirmID'].nunique() + rel_msgs.loc[in_T, 'FirmID'].isnull().sum()
    ###########################################################################
    ##### Number of users/messages/firms (N/M/F) in Race
    is_take = race_msgs['%sRaceRlvtType' % S].to_numpy() == 'Take Attempt'
    is_canc = race_msgs['%sRaceRlvtType' % S].to_numpy() == 'Cancel Attempt'
    is_success = race_msgs['%sRaceMsgOutcome' % S].to_numpy() == 'Success'
    is_fail = race_msgs['%sRaceMsgOutcome' % S].to_numpy() == 'Fail'
    is_IOC = (race_msgs['UnifiedMessageType'].to_numpy() == 'Gateway New Order (IOC)') & (race_msgs['TIF'].to_numpy() == 'IOC')
    is_Lim = race_msgs['UnifiedMessageType'].to_numpy() == 'Gateway New Order (Limit)'
    is_QuoteRelated = race_msgs['QuoteRelated'].to_numpy()
    is_at_P = race_msgs['%sRaceRlvtPriceLvlSigned' % S].to_numpy() == P_Signed   
    
    ### Firms 
    out['F_All'] = race_msgs['FirmID'].nunique() + race_msgs['FirmID'].isnull().sum()

    ### Users
    out['N_All'] = race_msgs['UserID'].nunique()

    ### Msgs
    out['M_All'] = race_msgs.shape[0]
    out['M_Canc'] = is_canc.sum()
    out['M_Take'] = is_take.sum()
    out['M_Take_IOC'] = (is_take & is_IOC).sum()
    out['M_Take_Lim'] = (is_take & is_Lim).sum()

    ### Successful race_msgs
    out['M_Success_All'] = is_success.sum()
    out['M_Success_Canc'] = (is_success & is_canc).sum()
    out['M_Success_Take'] = (is_success & is_take).sum()

    ### Failed race_msgs - Note that for all Cancels, the fail is a cancel reject or c/r reject.
    out['M_Fail_All'] = (is_fail).sum()
    out['M_Fail_Canc'] = (is_fail & is_canc).sum()
    out['M_Fail_Take'] = (is_fail & is_take).sum()
    out['M_Fail_Take_IOC'] = (is_fail & is_take & is_IOC).sum()
    out['M_Fail_Take_at_P'] = (is_fail & is_take & is_at_P).sum()

    ### Quote-Related Msgs: Flag if any race messages are quote related
    out['M_QR'] = is_QuoteRelated.sum()

    ### Capture fails that are specifically expired IOCs (rather than just trading at the wrong price)
    # By restricting the fail to the outcome group (rather than the outcome), we know that it failed regardless of
    # price. I.e. It failed because it expired, not because it traded at the wrong price.
    out['M_IOC_Expired'] = race_msgs.index[(race_msgs['%sRaceRlvtOutcomeGroup' % S] == 'Fail') &  \
                                           (race_msgs['%sRaceRlvtType' % S] == 'Take Attempt') & \
                                           (race_msgs['UnifiedMessageType'] == 'Gateway New Order (IOC)') & \
                                           (race_msgs['TIF'] == 'IOC')].nunique()
    ###########################################################################
    ##### Msgs Detailed Info
    ### S1 F1: message type, firm type, firm number and user number
    out['S1_Type'] = race_msgs.loc[is_success, '%sRaceRlvtType' % S].iloc[0]
    out['S1_FirmID'] = race_msgs.loc[is_success, 'FirmID'].iloc[0]
    out['S1_UserID'] = race_msgs.loc[is_success, 'UserID'].iloc[0]
    out['F1_Type'] = race_msgs.loc[is_fail, '%sRaceRlvtType' % S].iloc[0]
    out['F1_FirmID'] = race_msgs.loc[is_fail, 'FirmID'].iloc[0]
    out['F1_UserID'] = race_msgs.loc[is_fail, 'UserID'].iloc[0]
    
    ###########################################################################
    ##### Profits
    # Calculate price T into the future
    T_list = ['1ms', '10ms', '100ms', '1s', '10s', '30s', '60s', '100s']
    times = top.loc[(top.index >= ix_st) & (top['MessageTimestamp'] <= (time_end + np.timedelta64(100, 's')))]
    
    for t in T_list:
        # Missing midpoint will have np.nan raw_midpt_f, hence na for all profit measures
        in_t_T = times['MessageTimestamp'].to_numpy() <= time_st + np.timedelta64(pd.Timedelta(t))
        last_indix_in_T = int(in_t_T.nonzero()[0][-1]) # get the index of the last "True" in in_t_T
        raw_best_bid_f = times['BestBid'].iat[last_indix_in_T]
        raw_midpt_f =  times['MidPt'].iat[last_indix_in_T]
        raw_best_ask_f = times['BestAsk'].iat[last_indix_in_T]
        
        # Order book info
        out['BestBid_f_%s' % t] =  (1. / price_factor) * raw_best_bid_f
        out['BestAsk_f_%s' % t] =  (1. / price_factor) * raw_best_ask_f
        out['MidPt_f_%s' % t] = (1. / price_factor) *  raw_midpt_f
        
        # Share profits
        share_profits = (1. / price_factor) * (raw_midpt_f - float(P))
        out['Race_Profits_PerShare_%s' % t] = Sign * share_profits
        out['Race_Profits_PerShare_bps_%s' % t] = (Sign * share_profits * 10000.) / out['MidPt']
        out['Race_Profits_PerShare_Tx_%s' % t] = (1. / out['TickSize']) * Sign * share_profits

        # Race profits
        out['Race_Profits_DispDepth_%s' % t] = out['Depth_Disp'] * out['Race_Profits_PerShare_%s' % t]
        out['Race_Profits_TotalDepth_%s' % t] = out['Depth_Total'] * out['Race_Profits_PerShare_%s' % t]
        out['Race_Profits_ActiveQty_%s' % t] = out['Qty_Active'] * out['Race_Profits_PerShare_%s' % t]
    
    ###########################################################################
    ##### Loss Avoidance and Qty Traded Price Impact
    out['Eff_Spread_Paid_Race'] = (1./price_factor)*Sign*(float(P)-top_1['MidPt']) * out['Qty_Traded']
    out['Eff_Spread_PerShare_Race'] = (1./price_factor)*Sign*(float(P)-top_1['MidPt'])

    # Price Impact and Loss Avoidance
    T_list = ['1ms', '10ms', '100ms', '1s', '10s', '30s', '60s', '100s']
    times = top.loc[(top.index >= ix_st) & (top['MessageTimestamp'] <= (time_end + np.timedelta64(100, 's')))]
    for T in T_list:
        raw_midpt_f_T =  times.loc[times['MessageTimestamp'] <= (time_st + np.timedelta64(pd.Timedelta(T))), 'MidPt'].iloc[-1]
        out['LossAvoidance_%s' % T] = (1. / price_factor)*Sign*(raw_midpt_f_T - float(P)) * out['Qty_Cancelled']
        out['PriceImpact_Paid_Race_%s' % T] = (1. / price_factor)*Sign*(raw_midpt_f_T - top_1['MidPt']) * out['Qty_Traded']
        out['PriceImpact_PerShare_Race_%s' % T] = (1. / price_factor)*Sign*(raw_midpt_f_T - top_1['MidPt'])

###########################################################################
    ##### Pre-Race Stable BBO dummies
    T_list = ['10us','50us','100us', '500us', '1ms']
    for T in T_list:
        ## Preparation
        # Get the top-of-book data within T prior to the race
        # Note that ix_st is the index of the race starting msg
        # We want to select all msgs that are within T of the race start to get the prices T before the race
        # Get the msgs that are at least T before the race (whatever before race start - T)
        msgs_before_T_prerace = (msgs['MessageTimestamp'] < time_st - np.timedelta64(pd.Timedelta(T)))
        # Get the last msg that is at least T before the race 
        ix_T_prerace = msgs_before_T_prerace[msgs_before_T_prerace==True].last_valid_index()
        if ix_T_prerace is None:
            ix_T_prerace = 0
        within_T_prerace = (msgs.index >= ix_T_prerace) & (msgs.index <= ix_st)
        top_within_T = top[within_T_prerace]
        # Get the race side BBO prices from race start - T to race start
        race_bbo_prc_signed_within_T = top_within_T['Best%sSigned' % S]
        # Whether the BBO Quote is stable in price
        out['Stable_Prc_RaceBBO_since_%s_PreRace' % T] = np.all(race_bbo_prc_signed_within_T \
                                                                == race_bbo_prc_signed_within_T.iloc[-1])
        # Whether the BBO Quote has improved
        if race_bbo_prc_signed_within_T.iloc[0] < race_bbo_prc_signed_within_T.iloc[-1]:
            out['RaceBBO_Improved_since_%s_PreRace' % T] = 1
        elif race_bbo_prc_signed_within_T.iloc[0] > race_bbo_prc_signed_within_T.iloc[-1]:
            out['RaceBBO_Improved_since_%s_PreRace' % T] = -1
        else:
            out['RaceBBO_Improved_since_%s_PreRace' % T] = 0
    
    return (pd.Series(out))

######################
## Helper Functions ##
######################

def GetDepth(S, P, ix, total, depth):
    '''
    Given a side, price lvl and msg index, this function returns the depth at that price-side at the last depth change before
    the msg. If total == 'Total', we use hidden depth. Otherwise, displayed depth.
    
    Param:
        S: side
        P: price
        ix: idx of the msg
        total: ='Total' to include hidden depth
        depth: depth from order book construction
        
    Return:
        Sh: depth at that price-side at the last depth change 
            before the msg (# shares)
    '''
    # If hidden depth is desired, use _h as suffix
    hidden = ''
    if total == 'Total':
      hidden = '_h'
      
    Sh = 0
    if S == 'Ask':
        if P in depth['ask%s' % hidden].keys():
            depth_ask = depth['ask%s' % hidden][P]
            if min(depth_ask.index) <= ix:
                Sh = depth_ask.loc[depth_ask.index <= ix].iloc[-1]
    else:
        if P in depth['bid%s' % hidden].keys():
            depth_bid = depth['bid%s' % hidden][P]
            if min(depth_bid.index) <= ix:
                Sh = depth_bid.loc[depth_bid.index <= ix].iloc[-1]
    return (Sh)

def TradeNum_Vol(msg, *args):
    '''
    This function takes an inbound message and returns the sum of the executed volume and
    number of execution messages (trades) from all aggressive outbound fills related
    to that inbound. If at_P is true, this only includes quantity traded at the
    price/side in the function call (race P/S). If at_P is false, this provides all
    quantity traded because of that inbound message regardless of whether or not
    it is at the race P/S.
    
    Param:
          msg: The inbound message of interest
          args should include the three parameters (in order)
          S: side
          P: price level of the race
          at_P: If true, only include quantity traded at the price/side. 
             If false, provides all quantity traded because of that inbound message 
             regardless of whether or not it is at the race P/S.
          me, me_qr_ask, me_qr_bid: outbound dfs
             
    Global:
          Uses the three me (df) global variables generated by get_me_dfs()
  
    Return: 
          pd.Series([qty, trade]): A pandas series with two values, the sum of the executed depth and
          number of execution messages (trades) from all aggressive outbound fills related
          to that inbound.
          
    '''
    S, P, at_P, me, me_qr_ask, me_qr_bid = args[0], args[1], args[2], args[3], args[4], args[5] 
    qty, trade = np.nan, np.nan
    
    # If the function call does not require the output
    # to be calculated for a specific price/side, then 
    # use the side on which the message is race relevant
    # to ge the associated information (e.g. type) in
    # the next step. If the message is not ask race relevant
    # then it is either bid race relevant or 
    # it is not race relevant (and will not matter later)
    if not at_P:
        if msg['AskRaceRlvt']:
            S = 'Ask'
        else:
            S = 'Bid'    
    # If the message is a take attempt, then it may trade so continue the logic. 
    # otherwise, it will not end in a trade and can exit the function
    if (msg['%sRaceRlvtType' % S] == 'Take Attempt'):
        # If the message is a success, then it took depth (traded). Market orders
        # that fail also take depth. In these two cases, continue to calculate volume
        # and number of trades. Otherwise, exit the function
        if (msg['%sRaceRlvtOutcomeGroup' % S] == 'Race Price Dependent'):
            if not msg['QuoteRelated']:
                # If the UniqueOrderID and EventNum are in the outbound (me) data, then pull
                # the relevant trades and calculate the output. Otherwise, exit the function.
                # (This should always be the case)
                if (msg['UniqueOrderID'], msg['EventNum']) in me.index:
                    ev = me.loc[(msg['UniqueOrderID'], msg['EventNum'])]
                    # Calculate the number of trades and quantity traded at P or at all P depending on the function call
                    if at_P:
                        qty = ev.loc[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedPrice'] == P), 'ExecutedQty'].sum()
                        trade = len(ev[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedPrice'] == P) & (ev['ExecutedQty'] > 0)])
                    else:
                        qty = ev.loc[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}), 'ExecutedQty'].sum()
                        trade = len(ev[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedQty'] > 0)])                
                else:
                    qty = np.nan
            else:
                # If the message is quote-related.
                if S == 'Bid':
                    # If the UniqueOrderID and EventNum are in the outbound (me) data, then pull
                    # the relevant trades and calculate the output. Otherwise, exit the function.
                    # (This should always be the case)
                    if (msg['UniqueOrderID'], msg['AskEventNum']) in me_qr_ask.index:
                        ev = me_qr_ask.loc[(msg['UniqueOrderID'], msg['AskEventNum'])]
                        # Calculate the number of trades and quantity traded at P or at all P depending on the function call
                        if at_P:
                            qty = ev.loc[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedPrice'] == P), 'ExecutedQty'].sum()
                            trade = len(ev[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedPrice'] == P) & (ev['ExecutedQty'] > 0)])
                        else:
                            qty = ev.loc[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}), 'ExecutedQty'].sum()
                            trade = len(ev[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedQty'] > 0)])
                    else:
                        qty = np.nan
                elif S == 'Ask':
                    # If the UniqueOrderID and EventNum are in the outbound (me) data, then pull
                    # the relevant trades and calculate the output. Otherwise, exit the function.
                    # (This should always be the case, but is included as a backup to the function) [REMOVE? or COUNTER?]                          
                    if (msg['UniqueOrderID'], msg['BidEventNum']) in me_qr_bid.index:
                        ev = me_qr_bid.loc[(msg['UniqueOrderID'], msg['BidEventNum'])]
                        # Calculate the number of trades and quantity traded at P or at all P depending on the function call
                        if at_P:
                            qty = ev.loc[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedPrice'] == P), 'ExecutedQty'].sum()
                            trade = len(ev[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedPrice'] == P) & (ev['ExecutedQty'] > 0)])                            
                        else:
                            qty = ev.loc[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}), 'ExecutedQty'].sum()
                            trade = len(ev[ev['UnifiedMessageType'].isin({'ME: Partial Fill (A)', 'ME: Full Fill (A)'}) & (ev['ExecutedQty'] > 0)])                        
                    else:
                        qty = np.nan
    return pd.Series([qty, trade])
